# -*- coding: utf-8 -*-

from lxml import etree
from fake_useragent import UserAgent
import time
import random
import os
import requests
import shutil
import base64
import re
from fontTools.ttLib import TTFont

requests.packages.urllib3.disable_warnings()  # 忽略HTTPS安全警告

"""
58同城-出租
字体反爬
https://sh.58.com/chuzu/pn1/?PGTID=0d3090a7-0000-280e-1953-d4c631dd0689&ClickID=2
"""


class ChuZu():
    def __init__(self):
        ua = UserAgent(use_cache_server=False)  # 禁用服务器缓存
        self.headers = {
            "User-Agent": ua.random,
            "authority": "sh.58.com",
            "method": "GET",
            "path": "/chuzu/?PGTID=0d3090a7-0000-2fcd-3f28-496b220707d5&ClickID=2",
            "scheme": "https",
            "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
            "accept-encoding": "gzip, deflate, br",
            "accept-language": "zh-CN,zh;q=0.9",
            "cache-control": "max-age=0",
            "cookie": "f=n; commontopbar_new_city_info=2%7C%E4%B8%8A%E6%B5%B7%7Csh; commontopbar_ipcity=as%7C%E9%9E%8D%E5%B1%B1%7C0; userid360_xml=64FC99E7F2478084853E9E066145EEB7; time_create=1586495090770; f=n; id58=e87rZl5ocV8TR3pQBAUYAg==; 58home=as; city=as; 58tj_uuid=3e3620ba-fa09-4acd-b7cc-4b78d8c16adb; new_uv=1; utm_source=; spm=; init_refer=; als=0; xxzl_cid=408318ecd7604ac395bf4b6c7e455d12; xzuid=1fcd4c4a-8371-4fa2-bbbf-d824129c4726; wmda_new_uuid=1; wmda_uuid=9e5202b1fa991c5f18581089a0d7f7c4; wmda_session_id_11187958619315=1583903081494-3e06dad7-bcac-b2ad; wmda_visited_projects=%3B11187958619315; new_session=0; f=n; xxzl_deviceid=Hx6yR%2BJ9PIQYOjCUpdK4NkMSHNzPq3uuxlGpMuiLcM5Fn49Xkbz%2Fv3icpyRb29Af; ppStore_fingerprint=1EFEA998888D0872DEF818775BDD2B4E5612A25221321A83%EF%BC%BF1583903213879; xzfzqtoken=oLde0AMANzh20Oszp8P25hNy4XjXlQLBFL1nTU%2BOZ8rbhczU5g33WN930SDqzJb8in35brBb%2F%2FeSODvMgkQULA%3D%3D",
            "dnt": "1",
            "sec-fetch-mode": "navigate",
            "sec-fetch-site": "none",
            "sec-fetch-user": "?1",
            "upgrade-insecure-requests": "1"
        }
        # 字库
        self.font_dict = {
            "glyph00001": "0",
            "glyph00002": "1",
            "glyph00003": "2",
            "glyph00004": "3",
            "glyph00005": "4",
            "glyph00006": "5",
            "glyph00007": "6",
            "glyph00008": "7",
            "glyph00009": "8",
            "glyph00010": "9",
        }

    # 内容
    def get_contents(self):
        postData = {

        }
        pageNo = 1
        url = f"https://sh.58.com/chuzu/pn{pageNo}/?PGTID=0d3090a7-0000-23b2-c367-ebe3d25fb42d&ClickID=2"
        html = HttpUtils.do_request("GET", url, self.headers, postData)
        # 获取字体文件
        self.get_font_file(html.text)
        # 解析字体
        source_html = self.format_html_source(html.text)
        selector = etree.HTML(source_html)
        root = selector.xpath('//li[@class="house-cell"]')
        for s in root:
            dict_data = dict()
            dict_data['url'] = s.xpath('./div[@class="des"]/h2/a/@href')[0]  # 链接
            dict_data['title'] = s.xpath('./div[@class="des"]/h2/a/text()')[0].replace("\n", "").strip()  # 标题
            dict_data['type'] = s.xpath('./div[@class="des"]/p[@class="room"]/text()')[0]\
                .replace(" ", "")  # 类型
            dict_data['area'] = "|".join(s.xpath('./div[@class="des"]/p[@class="infor"]/a/text()'))  # 区域
            dict_data['price'] = "".join(s.xpath('./div[@class="list-li-right"]/div[@class="money"]//text()'))\
                .replace("\n", "").strip()  # 价格
            dict_data['source'] = s.xpath('./div[@class="des"]/div[@class="jjr"]/text()')[0].replace("\n", "").strip()  # 来源
            dict_data['company'] = s.xpath('./div[@class="des"]/div[@class="jjr"]/span/span[1]/text()')[0].replace("\n", "")\
                .strip()  # 公司名称
            dict_data['lsr'] = s.xpath('./div[@class="des"]/div[@class="jjr"]/span/span[2]/text()')[0].replace("\n", "")\
                .strip()  # 联系人
            print(str(dict_data))
            self.get_detail(dict_data)

    # 详情信息
    def get_detail(self, dict_data):
        postData = {

        }
        html = HttpUtils.do_request("GET", dict_data['url'], self.headers, postData)
        # 获取字体文件
        self.get_font_file(html.text)
        # 解析字体
        source_html = self.format_html_source(html.text)
        selector = etree.HTML(source_html)
        dict_data['title_1'] = selector.xpath('//div[@class="house-title"]/h1/text()')[0]
        dict_data['price_1'] = "".join(selector.xpath('//span[@class="c_ff552e"]//text()'))\
            .replace("\n", "").replace(" ", "")  # 价格
        dict_data['pay_type'] = selector.xpath('//span[@class="instructions"]/text()')[0]  # 押一付一
        dict_data['type_1'] = selector.xpath('//ul[@class="f14"]/li[1]/span[2]/text()')[0]  # 租赁方式
        dict_data['hourse_type'] = selector.xpath('//ul[@class="f14"]/li[2]/span[2]/text()')[0].replace(" ", "")\
            .strip()  # 房屋类型
        dict_data['chaoxiang'] = selector.xpath('//ul[@class="f14"]/li[3]/span[2]/text()')[0].strip()  # 朝向楼层
        dict_data['area_2'] = selector.xpath('//ul[@class="f14"]/li[4]/span[2]//text()')[0].strip()  # 所在小区
        dict_data['area_1'] = "".join(selector.xpath('//ul[@class="f14"]/li[5]/span[2]//text()')).replace("\n", "")\
            .replace(" ", "").strip()  # 所属区域
        dict_data['add'] = "".join(selector.xpath('//ul[@class="f14"]/li[6]/span[2]//text()')).replace("\n", "").strip()  # 地址
        dict_data['tel'] = selector.xpath('//p[@class="phone-num strongbox"]/text()')[0].replace("\n", "").strip()  # 电话
        print(str(dict_data))


    # 获取字体文件,保存TTFont对象
    def get_font_file(self, html):
        try:
            b64 = re.findall(r"base64,(.*?)\'", html, re.S)[0]
            res = base64.b64decode(b64)
            with open("fonts/online_font.ttf", "wb") as f:
                f.write(res)
            self.online_font = TTFont("fonts/online_font.ttf")
            self.online_font.saveXML("./fonts/online.xml")
        except IndexError:
            return

    # 解析字体信息。返回解析后html
    def format_html_source(self, html):
        assert self.online_font, "必须创建字体对象"
        assert os.path.exists("fonts/online.xml"), "请先获取到字体文件。"
        with open("fonts/online.xml", "rb") as f:
            file_data = f.read()
        online_uni_list = self.online_font.getGlyphOrder()[1:]
        file_selector = etree.HTML(file_data)
        for uni2 in online_uni_list:
            code = file_selector.xpath("//cmap//map[@name='{}']/@code".format(uni2))[0]
            dd = "&#x" + code[2:].lower() + ";"
            if dd in html:
                html = html.replace(dd, self.font_dict[uni2])
        return html


if __name__ == '__main__':
    cuzu = ChuZu()
    cuzu.get_contents()